{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#| default_exp phm2008"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PHM2008 dataset\n",
    "\n",
    "> Download the PHM2008 dataset.\n",
    "\n",
    "Prognosis and Health Management 2008 challenge dataset. This dataset used the Commercial Modular Aero-Propulsion System Simulation to recreate the degradation process of turbofan engines for different aircraft with varying wear and manufacturing starting under normal conditions. The training dataset consists of complete run-to-failure simulations, while the test dataset comprises sequences before failure.\n",
    "\n",
    "[Saxena, A., Goebel, K., Simon, D.,&Eklund, N. (2008). \"Damage propagation modeling for aircraft engine run-to-failure simulation\". International conference on prognostics and health management.](https://ntrs.nasa.gov/api/citations/20090029214/downloads/20090029214.pdf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "import os\n",
    "from dataclasses import dataclass\n",
    "from typing import Tuple\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from datasetsforecast.utils import extract_file, download_file, Info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "@dataclass\n",
    "class FD001:\n",
    "    seasonality: int = 1\n",
    "    horizon: int = 1\n",
    "    freq: str = 'None'\n",
    "    train_file: str = 'train_FD001.txt'\n",
    "    test_file: str = 'test_FD001.txt'\n",
    "    rul_file: str = 'RUL_FD001.txt'\n",
    "    n_ts: int = 100\n",
    "    n_test: int = 100\n",
    "\n",
    "@dataclass\n",
    "class FD002:\n",
    "    seasonality: int = 1\n",
    "    horizon: int = 1\n",
    "    freq: str = 'None'\n",
    "    train_file: str = 'train_FD002.txt'\n",
    "    test_file: str = 'test_FD002.txt'\n",
    "    rul_file: str = 'RUL_FD002.txt'\n",
    "    n_ts: int = 260\n",
    "    n_test: int = 259\n",
    "\n",
    "@dataclass\n",
    "class FD003:\n",
    "    seasonality: int = 1\n",
    "    horizon: int = 1\n",
    "    freq: str = 'None'\n",
    "    train_file: str = 'train_FD003.txt'\n",
    "    test_file: str = 'test_FD003.txt'\n",
    "    rul_file: str = 'RUL_FD003.txt'\n",
    "    n_ts: int = 100\n",
    "    n_test: int = 100\n",
    "\n",
    "@dataclass\n",
    "class FD004:\n",
    "    seasonality: int = 1\n",
    "    horizon: int = 8\n",
    "    freq: str = 'None'\n",
    "    train_file: str = 'train_FD004.txt'\n",
    "    test_file: str = 'test_FD004.txt'\n",
    "    rul_file: str = 'RUL_FD004.txt'\n",
    "    n_ts: int = 249\n",
    "    n_test: int = 248"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "PHM2008Info = Info((FD001, FD002, FD003, FD004))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#| export\n",
    "@dataclass\n",
    "class PHM2008:\n",
    "\n",
    "    #source_url = 'https://forecasters.org/data/m3comp/M3C.xls'\n",
    "    source_url = 'https://www.dropbox.com/s/1od45uh37tplxt7/CMAPSSData.zip?dl=1'\n",
    "\n",
    "    @staticmethod\n",
    "    def download(directory: str) -> None:\n",
    "        \"\"\"\n",
    "        Download PHM2008 Dataset.\n",
    "        \n",
    "        Parameters\n",
    "        ----------\n",
    "        directory: str\n",
    "            Directory path to download dataset.\n",
    "        \"\"\"\n",
    "        path = f'{directory}/phm2008/'\n",
    "        if not os.path.exists(path):\n",
    "            download_file(path, PHM2008.source_url)\n",
    "        if not os.path.isdir(f'{directory}/phm2008/CMAPSSData'):\n",
    "            extract_file(f'{path}CMAPSSData.zip', path)\n",
    "\n",
    "    @staticmethod\n",
    "    def load(directory: str, group: str, clip_rul: bool=True) -> Tuple[pd.DataFrame, pd.DataFrame]:\n",
    "        \"\"\"\n",
    "        Downloads and loads M3 data.\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        directory: str\n",
    "            Directory where data will be downloaded.\n",
    "        group: str\n",
    "            Group name.\n",
    "            Allowed groups: 'FD001', 'FD002', 'FD003', 'FD004'.\n",
    "        clip_rul: bool\n",
    "            Wether or not upper bound the remaining useful life to 125.\n",
    "\n",
    "        Returns\n",
    "        -------\n",
    "        df: pd.DataFrame\n",
    "            Target time series with columns ['unique_id', 'ds', 'y', 'exogenous'].\n",
    "        \"\"\"\n",
    "        def _add_remaining_useful_life(df):\n",
    "            # Get the total number of cycles for each unit\n",
    "            grouped_by_unit = df.groupby(by=\"unit_nr\")\n",
    "            max_cycle = grouped_by_unit[\"time_cycles\"].max()\n",
    "\n",
    "            # Merge the max cycle back into the original frame\n",
    "            result_frame = df.merge(max_cycle.to_frame(name='max_cycle'), left_on='unit_nr', right_index=True)\n",
    "\n",
    "            # Calculate remaining useful life for each row (piece-wise Linear)\n",
    "            remaining_useful_life = result_frame[\"max_cycle\"] - result_frame[\"time_cycles\"]\n",
    "            result_frame[\"RUL\"] = remaining_useful_life\n",
    "\n",
    "            # drop max_cycle as it's no longer needed\n",
    "            result_frame = result_frame.drop(\"max_cycle\", axis=1)\n",
    "\n",
    "            return result_frame\n",
    "\n",
    "        PHM2008.download(directory)\n",
    "        \n",
    "        path = f'{directory}/phm2008/CMAPSSData'\n",
    "        group = PHM2008Info.get_group(group)\n",
    "        \n",
    "        # define column names for easy indexing\n",
    "        index_names = ['unit_nr', 'time_cycles']\n",
    "        setting_names = ['setting_1', 'setting_2', 'setting_3']\n",
    "        sensor_names = ['s_{}'.format(i) for i in range(1, 22)]\n",
    "        col_names = index_names + setting_names + sensor_names\n",
    "\n",
    "        # read data\n",
    "        train = pd.read_csv((f'{path}/{group.train_file}'), \n",
    "                            sep='\\s+', header=None, names=col_names)\n",
    "        test = pd.read_csv((f'{path}/{group.test_file}'), \n",
    "                        sep='\\s+', header=None, names=col_names)\n",
    "        y_test = pd.read_csv((f'{path}/{group.rul_file}'), \n",
    "                            sep='\\s+', header=None, names=['RUL'])\n",
    "        \n",
    "\n",
    "        # drop non-informative features in training set\n",
    "        drop_sensors = ['s_1', 's_5', 's_6', 's_10', 's_16', 's_18', 's_19']\n",
    "        drop_labels = setting_names + drop_sensors\n",
    "        train.drop(labels=drop_labels, axis=1, inplace=True)\n",
    "\n",
    "        # Add piece-wise target remaining useful life\n",
    "        # in the paper the MAX RUL is mentioned as 125\n",
    "        train = _add_remaining_useful_life(train)\n",
    "\n",
    "        # Training set\n",
    "        Y_train_df = train.rename(columns={'unit_nr': 'unique_id', 'time_cycles': 'ds',\n",
    "                                           'RUL': 'y'})\n",
    "\n",
    "        # drop non-informative features in testing set\n",
    "        test.drop(labels=drop_labels, axis=1, inplace=True)\n",
    "\n",
    "        # Testing set\n",
    "        Y_test_df = test.rename(columns={'unit_nr': 'unique_id', 'time_cycles': 'ds'})\n",
    "        \n",
    "        # Only last RUL is available, complementing Y_test with linearly decreasing RUL\n",
    "        count_ds = Y_test_df[['unique_id', 'ds']].groupby('unique_id').count().reset_index()\n",
    "        ruls_list= []\n",
    "        for index, row in count_ds.iterrows():\n",
    "            ruls_list.append(np.arange(row.ds+y_test.RUL.values[index]-1, \n",
    "                                    y_test.RUL.values[index]-1, -1))\n",
    "        ruls_list = np.concatenate(ruls_list)\n",
    "        Y_test_df['y'] = ruls_list\n",
    "\n",
    "        # Check that last RULs match\n",
    "        last_rul = Y_test_df[['unique_id', 'y']].groupby('unique_id').last().reset_index()\n",
    "        last_rul = last_rul['y'].values\n",
    "        assert np.array_equal(last_rul, y_test.RUL.values)\n",
    "\n",
    "        if clip_rul:\n",
    "            Y_train_df['y'].clip(upper=125, inplace=True)\n",
    "            Y_test_df['y'].clip(upper=125, inplace=True)\n",
    "\n",
    "        return Y_train_df, Y_test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#| hide\n",
    "# Unit tests\n",
    "for group, meta in PHM2008Info:\n",
    "    # Checking that load works\n",
    "    Y_train_df, Y_test_df = PHM2008.load(directory='data', group=group)\n",
    "    \n",
    "    # Checking that train series match info's train series\n",
    "    unique_elements = Y_train_df.groupby(['unique_id', 'ds']).size()\n",
    "    unique_ts = Y_train_df.groupby('unique_id').size()\n",
    "    assert (unique_elements != 1).sum() == 0, f'Duplicated records found: {group}'\n",
    "    assert unique_ts.shape[0] == meta.n_ts, f'Number of time series not match: {group}'\n",
    "\n",
    "    # Checking that test series match info's test series\n",
    "    unique_elements = Y_test_df.groupby(['unique_id', 'ds']).size()\n",
    "    unique_ts = Y_test_df.groupby('unique_id').size()\n",
    "    assert (unique_elements != 1).sum() == 0, f'Duplicated records found: {group}'\n",
    "    assert unique_ts.shape[0] == meta.n_test, f'Number of time series not match: {group}'\n",
    "\n",
    "    # Assert no nans\n",
    "    assert Y_train_df.isnull().values.any()==False\n",
    "    assert Y_test_df.isnull().values.any()==False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "neuralforecast",
   "language": "python",
   "name": "neuralforecast"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
